{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:36:57.814597850Z",
     "start_time": "2024-06-04T19:36:57.412383980Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "import json\n",
    "from typing import List, Optional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_trait(trait):\n",
    "    trait = '_'.join(trait.split())\n",
    "    normalized_trait = ''.join(trait.split(\"'\"))\n",
    "    return normalized_trait\n",
    "\n",
    "def normalize_gene_symbols(gene_symbols: List[str]) -> List[Optional[str]]:\n",
    "    \"\"\"Use gene synonym information extracted from the NCBI Gene database to normalize gene symbols in a list, and\n",
    "    return a list of normalized symbols. Unmatched symbols are converted to None.\n",
    "    \"\"\"\n",
    "    with open(\"./metadata/gene_synonym.json\", \"r\") as f:\n",
    "        synonym_dict = json.load(f)\n",
    "    return [synonym_dict.get(g) for g in gene_symbols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Extract genes from NCBI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Get trait-condition pairs and get gene IoUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:36:58.766036461Z",
     "start_time": "2024-06-04T19:36:58.757777848Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(\"../metadata/task_info.json\", \"r\") as f:\n",
    "    info = json.load(f)\n",
    "t2g = {k: v['related_genes'] for k, v in info.items()}\n",
    "traits = sorted(list(t2g.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "132"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(traits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "trait_df = pd.read_csv('../download/new_task.csv')\n",
    "all_cancers = trait_df.loc[trait_df['Type'] == '1. Cancer and Oncology-Related Disorders', 'Trait'].tolist()\n",
    "all_cancers = [normalize_trait(c) for c in all_cancers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38\n",
      "['Breast_Cancer', 'Kidney_Papillary_Cell_Carcinoma', 'Kidney_Clear_Cell_Carcinoma', 'Head_and_Neck_Cancer', 'Glioblastoma', 'Esophageal_Cancer', 'Endometrioid_Cancer', 'Bladder_Cancer', 'Retinoblastoma', 'Bile_Duct_Cancer', 'Adrenocortical_Cancer', 'Acute_Myeloid_Leukemia', 'X-Linked_Lymphoproliferative_Syndrome', 'Von_Hippel_Lindau', 'Liver_Cancer', 'Lower_Grade_Glioma', 'Large_B-cell_Lymphoma', 'Mesothelioma', 'Lung_Cancer', 'Uterine_Corpus_Endometrial_Carcinoma', 'Uterine_Carcinosarcoma', 'Thymoma', 'Thyroid_Cancer', 'Melanoma', 'Stomach_Cancer', 'Sarcoma', 'Testicular_Cancer', 'Rectal_Cancer', 'Prostate_Cancer', 'Pheochromocytoma_and_Paraganglioma', 'Pancreatic_Cancer', 'Ovarian_Cancer', 'Ocular_Melanomas', 'Li-Fraumeni_Syndrome', 'Cervical_Cancer', 'Colon_and_Rectal_Cancer', 'Kidney_Chromophobe', 'lower_grade_glioma_and_glioblastoma']\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "all_cancers =\n",
    "['Breast_Cancer', 'Kidney_Papillary_Cell_Carcinoma', 'Kidney_Clear_Cell_Carcinoma', 'Head_and_Neck_Cancer', \n",
    "'Glioblastoma', 'Esophageal_Cancer', 'Endometrioid_Cancer', 'Bladder_Cancer', 'Retinoblastoma', 'Bile_Duct_Cancer', \n",
    "'Adrenocortical_Cancer', 'Acute_Myeloid_Leukemia', 'X-Linked_Lymphoproliferative_Syndrome', 'Von_Hippel_Lindau', \n",
    "'Liver_Cancer', 'Lower_Grade_Glioma', 'Large_B-cell_Lymphoma', 'Mesothelioma', 'Lung_Cancer', \n",
    "'Uterine_Corpus_Endometrial_Carcinoma', 'Uterine_Carcinosarcoma', 'Thymoma', 'Thyroid_Cancer', 'Melanoma', \n",
    "'Stomach_Cancer', 'Sarcoma', 'Testicular_Cancer', 'Rectal_Cancer', 'Prostate_Cancer', 'Pheochromocytoma_and_Paraganglioma', \n",
    "'Pancreatic_Cancer', 'Ovarian_Cancer', 'Ocular_Melanomas', 'Li-Fraumeni_Syndrome', 'Cervical_Cancer', \n",
    "'Colon_and_Rectal_Cancer', 'Kidney_Chromophobe', 'lower_grade_glioma_and_glioblastoma']\n",
    "len(all_cancers) = 38\n",
    "\"\"\"\n",
    "print(len(all_cancers))\n",
    "print(all_cancers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\n",
      "['Acute_Myeloid_Leukemia', 'Adrenocortical_Cancer', 'Bile_Duct_Cancer', 'Bladder_Cancer', 'Breast_Cancer', 'Cervical_Cancer', 'Colon_and_Rectal_Cancer', 'Endometrioid_Cancer', 'Esophageal_Cancer', 'Glioblastoma', 'Head_and_Neck_Cancer', 'Kidney_Chromophobe', 'Kidney_Clear_Cell_Carcinoma', 'Kidney_Papillary_Cell_Carcinoma', 'Large_B-cell_Lymphoma', 'Liver_Cancer', 'Lower_Grade_Glioma', 'Lung_Cancer', 'Melanoma', 'Mesothelioma', 'Ocular_Melanomas', 'Ovarian_Cancer', 'Pancreatic_Cancer', 'Pheochromocytoma_and_Paraganglioma', 'Prostate_Cancer', 'Rectal_Cancer', 'Retinoblastoma', 'Sarcoma', 'Stomach_Cancer', 'Testicular_Cancer', 'Thymoma', 'Thyroid_Cancer', 'Uterine_Carcinosarcoma', 'Uterine_Corpus_Endometrial_Carcinoma', 'lower_grade_glioma_and_glioblastoma']\n"
     ]
    }
   ],
   "source": [
    "cancer_traits = [c for c in traits if ((c in all_cancers) and (c not in ['Von_Hippel_Lindau', 'X-Linked_Lymphoproliferative_Syndrome']))]\n",
    "print(len(cancer_traits))\n",
    "print(cancer_traits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:37:06.554528694Z",
     "start_time": "2024-06-04T19:37:06.132639434Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = []\n",
    "for t in traits:\n",
    "    for s in traits + ['Age', 'Gender']:\n",
    "        if t == s: continue\n",
    "        if s in ['Age', 'Gender']: \n",
    "            iou = 1.0\n",
    "        else:\n",
    "            len_union = len(set(t2g[t]).union(set(t2g[s])))\n",
    "            if len_union == 0: continue\n",
    "            iou = len(set(t2g[t]).intersection(set(t2g[s]))) / len_union\n",
    "        data.append({'Trait': t, 'Condition': s, 'IoU': iou})\n",
    "rel = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trait</th>\n",
       "      <th>Condition</th>\n",
       "      <th>IoU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Adrenocortical_Cancer</td>\n",
       "      <td>0.018519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Age-Related_Macular_Degeneration</td>\n",
       "      <td>0.146132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Alcohol_Flush_Reaction</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Allergies</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Alopecia</td>\n",
       "      <td>0.095890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17521</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Von_Willebrand_Disease</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17522</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Werner_Syndrome</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17523</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>X-Linked_Lymphoproliferative_Syndrome</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17524</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Age</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17525</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17526 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Trait  \\\n",
       "0                   Acute_Myeloid_Leukemia   \n",
       "1                   Acute_Myeloid_Leukemia   \n",
       "2                   Acute_Myeloid_Leukemia   \n",
       "3                   Acute_Myeloid_Leukemia   \n",
       "4                   Acute_Myeloid_Leukemia   \n",
       "...                                    ...   \n",
       "17521  lower_grade_glioma_and_glioblastoma   \n",
       "17522  lower_grade_glioma_and_glioblastoma   \n",
       "17523  lower_grade_glioma_and_glioblastoma   \n",
       "17524  lower_grade_glioma_and_glioblastoma   \n",
       "17525  lower_grade_glioma_and_glioblastoma   \n",
       "\n",
       "                                   Condition       IoU  \n",
       "0                      Adrenocortical_Cancer  0.018519  \n",
       "1           Age-Related_Macular_Degeneration  0.146132  \n",
       "2                     Alcohol_Flush_Reaction  0.000000  \n",
       "3                                  Allergies  0.038462  \n",
       "4                                   Alopecia  0.095890  \n",
       "...                                      ...       ...  \n",
       "17521                 Von_Willebrand_Disease  0.000000  \n",
       "17522                        Werner_Syndrome  0.000000  \n",
       "17523  X-Linked_Lymphoproliferative_Syndrome  0.000000  \n",
       "17524                                    Age  1.000000  \n",
       "17525                                 Gender  1.000000  \n",
       "\n",
       "[17526 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:37:22.654936834Z",
     "start_time": "2024-06-04T19:37:22.602365235Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 回头修订一下。\n",
    "\n",
    "condition_only = ['Vitamin_D_Levels', 'LDL_Cholesterol_Levels']\n",
    "male_traits = ['Prostate_Cancer', 'Testicular_Cancer']\n",
    "female_traits = ['Cervical_Cancer', 'Endometriosis', 'Endometrioid_Cancer', 'Uterine_Carcinosarcoma', 'Uterine_Corpus_Endometrial_Carcinoma', 'Ovarian_Cancer', 'Polycystic_Ovary_Syndrome'] \n",
    "gender_traits = male_traits + female_traits\n",
    "condition_all = ['Obesity', 'Hypertension', 'Type_2_Diabetes']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert all(t in traits for t in gender_traits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:37:23.200319084Z",
     "start_time": "2024-06-04T19:37:23.165781124Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trait</th>\n",
       "      <th>Condition</th>\n",
       "      <th>IoU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Adrenocortical_Cancer</td>\n",
       "      <td>0.018519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Age-Related_Macular_Degeneration</td>\n",
       "      <td>0.146132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Alcohol_Flush_Reaction</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Allergies</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Alopecia</td>\n",
       "      <td>0.095890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17521</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Von_Willebrand_Disease</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17522</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Werner_Syndrome</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17523</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>X-Linked_Lymphoproliferative_Syndrome</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17524</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Age</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17525</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17251 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Trait  \\\n",
       "0                   Acute_Myeloid_Leukemia   \n",
       "1                   Acute_Myeloid_Leukemia   \n",
       "2                   Acute_Myeloid_Leukemia   \n",
       "3                   Acute_Myeloid_Leukemia   \n",
       "4                   Acute_Myeloid_Leukemia   \n",
       "...                                    ...   \n",
       "17521  lower_grade_glioma_and_glioblastoma   \n",
       "17522  lower_grade_glioma_and_glioblastoma   \n",
       "17523  lower_grade_glioma_and_glioblastoma   \n",
       "17524  lower_grade_glioma_and_glioblastoma   \n",
       "17525  lower_grade_glioma_and_glioblastoma   \n",
       "\n",
       "                                   Condition       IoU  \n",
       "0                      Adrenocortical_Cancer  0.018519  \n",
       "1           Age-Related_Macular_Degeneration  0.146132  \n",
       "2                     Alcohol_Flush_Reaction  0.000000  \n",
       "3                                  Allergies  0.038462  \n",
       "4                                   Alopecia  0.095890  \n",
       "...                                      ...       ...  \n",
       "17521                 Von_Willebrand_Disease  0.000000  \n",
       "17522                        Werner_Syndrome  0.000000  \n",
       "17523  X-Linked_Lymphoproliferative_Syndrome  0.000000  \n",
       "17524                                    Age  1.000000  \n",
       "17525                                 Gender  1.000000  \n",
       "\n",
       "[17251 rows x 3 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rel = rel[~((rel['Trait'].isin(gender_traits)) & (rel['Condition'] == 'Gender'))]\n",
    "rel = rel[~rel['Trait'].isin(condition_only)]\n",
    "rel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:37:27.928814904Z",
     "start_time": "2024-06-04T19:37:27.886976552Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trait</th>\n",
       "      <th>Condition</th>\n",
       "      <th>IoU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Adrenocortical_Cancer</td>\n",
       "      <td>0.018519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Age-Related_Macular_Degeneration</td>\n",
       "      <td>0.146132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Alcohol_Flush_Reaction</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Allergies</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Alopecia</td>\n",
       "      <td>0.095890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17521</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Von_Willebrand_Disease</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17522</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Werner_Syndrome</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17523</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>X-Linked_Lymphoproliferative_Syndrome</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17524</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Age</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17525</th>\n",
       "      <td>lower_grade_glioma_and_glioblastoma</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17223 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Trait  \\\n",
       "0                   Acute_Myeloid_Leukemia   \n",
       "1                   Acute_Myeloid_Leukemia   \n",
       "2                   Acute_Myeloid_Leukemia   \n",
       "3                   Acute_Myeloid_Leukemia   \n",
       "4                   Acute_Myeloid_Leukemia   \n",
       "...                                    ...   \n",
       "17521  lower_grade_glioma_and_glioblastoma   \n",
       "17522  lower_grade_glioma_and_glioblastoma   \n",
       "17523  lower_grade_glioma_and_glioblastoma   \n",
       "17524  lower_grade_glioma_and_glioblastoma   \n",
       "17525  lower_grade_glioma_and_glioblastoma   \n",
       "\n",
       "                                   Condition       IoU  \n",
       "0                      Adrenocortical_Cancer  0.018519  \n",
       "1           Age-Related_Macular_Degeneration  0.146132  \n",
       "2                     Alcohol_Flush_Reaction  0.000000  \n",
       "3                                  Allergies  0.038462  \n",
       "4                                   Alopecia  0.095890  \n",
       "...                                      ...       ...  \n",
       "17521                 Von_Willebrand_Disease  0.000000  \n",
       "17522                        Werner_Syndrome  0.000000  \n",
       "17523  X-Linked_Lymphoproliferative_Syndrome  0.000000  \n",
       "17524                                    Age  1.000000  \n",
       "17525                                 Gender  1.000000  \n",
       "\n",
       "[17223 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rel = rel[~(rel['Trait'].isin(male_traits) & rel['Condition'].isin(female_traits))]\n",
    "rel = rel[~(rel['Trait'].isin(female_traits) & rel['Condition'].isin(male_traits))]\n",
    "rel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:37:33.792487082Z",
     "start_time": "2024-06-04T19:37:33.763778059Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trait</th>\n",
       "      <th>Condition</th>\n",
       "      <th>IoU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Type_2_Diabetes</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Von_Willebrand_Disease</td>\n",
       "      <td>Age</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Von_Willebrand_Disease</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Von_Hippel_Lindau</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Von_Hippel_Lindau</td>\n",
       "      <td>Age</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17218</th>\n",
       "      <td>Kidney_Clear_Cell_Carcinoma</td>\n",
       "      <td>Rectal_Cancer</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17219</th>\n",
       "      <td>Kidney_Clear_Cell_Carcinoma</td>\n",
       "      <td>Psoriatic_Arthritis</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17220</th>\n",
       "      <td>Post-Traumatic_Stress_Disorder</td>\n",
       "      <td>Aniridia</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17221</th>\n",
       "      <td>Post-Traumatic_Stress_Disorder</td>\n",
       "      <td>Angelman_Syndrome</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17222</th>\n",
       "      <td>Kidney_Clear_Cell_Carcinoma</td>\n",
       "      <td>Obsessive-Compulsive_Disorder</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17223 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Trait                      Condition  IoU\n",
       "0                     Type_2_Diabetes                         Gender  1.0\n",
       "1              Von_Willebrand_Disease                            Age  1.0\n",
       "2              Von_Willebrand_Disease                         Gender  1.0\n",
       "3                   Von_Hippel_Lindau                         Gender  1.0\n",
       "4                   Von_Hippel_Lindau                            Age  1.0\n",
       "...                               ...                            ...  ...\n",
       "17218     Kidney_Clear_Cell_Carcinoma                  Rectal_Cancer  0.0\n",
       "17219     Kidney_Clear_Cell_Carcinoma            Psoriatic_Arthritis  0.0\n",
       "17220  Post-Traumatic_Stress_Disorder                       Aniridia  0.0\n",
       "17221  Post-Traumatic_Stress_Disorder              Angelman_Syndrome  0.0\n",
       "17222     Kidney_Clear_Cell_Carcinoma  Obsessive-Compulsive_Disorder  0.0\n",
       "\n",
       "[17223 rows x 3 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rel = rel.sort_values(by='IoU', ascending=False).reset_index().drop(columns=['index'])\n",
    "rel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:44:53.065631021Z",
     "start_time": "2024-06-04T19:44:53.002910008Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trait</th>\n",
       "      <th>Condition</th>\n",
       "      <th>IoU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Type_2_Diabetes</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Von_Willebrand_Disease</td>\n",
       "      <td>Age</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Von_Willebrand_Disease</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Von_Hippel_Lindau</td>\n",
       "      <td>Gender</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Von_Hippel_Lindau</td>\n",
       "      <td>Age</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16961</th>\n",
       "      <td>Kidney_Chromophobe</td>\n",
       "      <td>Obesity</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17003</th>\n",
       "      <td>Kidney_Chromophobe</td>\n",
       "      <td>Type_2_Diabetes</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17083</th>\n",
       "      <td>Kidney_Clear_Cell_Carcinoma</td>\n",
       "      <td>Type_2_Diabetes</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17099</th>\n",
       "      <td>Multiple_Endocrine_Neoplasia_Type_2</td>\n",
       "      <td>Type_2_Diabetes</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17212</th>\n",
       "      <td>Kidney_Clear_Cell_Carcinoma</td>\n",
       "      <td>Obesity</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>856 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Trait        Condition  IoU\n",
       "0                          Type_2_Diabetes           Gender  1.0\n",
       "1                   Von_Willebrand_Disease              Age  1.0\n",
       "2                   Von_Willebrand_Disease           Gender  1.0\n",
       "3                        Von_Hippel_Lindau           Gender  1.0\n",
       "4                        Von_Hippel_Lindau              Age  1.0\n",
       "...                                    ...              ...  ...\n",
       "16961                   Kidney_Chromophobe          Obesity  0.0\n",
       "17003                   Kidney_Chromophobe  Type_2_Diabetes  0.0\n",
       "17083          Kidney_Clear_Cell_Carcinoma  Type_2_Diabetes  0.0\n",
       "17099  Multiple_Endocrine_Neoplasia_Type_2  Type_2_Diabetes  0.0\n",
       "17212          Kidney_Clear_Cell_Carcinoma          Obesity  0.0\n",
       "\n",
       "[856 rows x 3 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected = rel[(rel['IoU'] >= 0.5) | rel['Condition'].isin(condition_all)]\n",
    "selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trait</th>\n",
       "      <th>Condition</th>\n",
       "      <th>IoU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>Lung_Cancer</td>\n",
       "      <td>Breast_Cancer</td>\n",
       "      <td>0.851852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>Breast_Cancer</td>\n",
       "      <td>Lung_Cancer</td>\n",
       "      <td>0.851852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>Prostate_Cancer</td>\n",
       "      <td>Lung_Cancer</td>\n",
       "      <td>0.801802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>Lung_Cancer</td>\n",
       "      <td>Prostate_Cancer</td>\n",
       "      <td>0.801802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>Melanoma</td>\n",
       "      <td>Ovarian_Cancer</td>\n",
       "      <td>0.785714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>Thyroid_Cancer</td>\n",
       "      <td>Bladder_Cancer</td>\n",
       "      <td>0.503759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Bladder_Cancer</td>\n",
       "      <td>0.503759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502</th>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>Glioblastoma</td>\n",
       "      <td>0.503759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>505</th>\n",
       "      <td>Bladder_Cancer</td>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>0.503759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>Glioblastoma</td>\n",
       "      <td>Acute_Myeloid_Leukemia</td>\n",
       "      <td>0.503759</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Trait               Condition       IoU\n",
       "251             Lung_Cancer           Breast_Cancer  0.851852\n",
       "252           Breast_Cancer             Lung_Cancer  0.851852\n",
       "253         Prostate_Cancer             Lung_Cancer  0.801802\n",
       "254             Lung_Cancer         Prostate_Cancer  0.801802\n",
       "255                Melanoma          Ovarian_Cancer  0.785714\n",
       "..                      ...                     ...       ...\n",
       "500          Thyroid_Cancer          Bladder_Cancer  0.503759\n",
       "501  Acute_Myeloid_Leukemia          Bladder_Cancer  0.503759\n",
       "502  Acute_Myeloid_Leukemia            Glioblastoma  0.503759\n",
       "505          Bladder_Cancer  Acute_Myeloid_Leukemia  0.503759\n",
       "507            Glioblastoma  Acute_Myeloid_Leukemia  0.503759\n",
       "\n",
       "[78 rows x 3 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected[(selected['Trait'].isin(cancer_traits)) & (selected['Condition'].isin(cancer_traits))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Please help me write the code to aggregate the conditions corresponding to each trait in the 'selected' DataFrame into a list\n",
    "# Originally, t2g is a dictionary with trait names as keys and a list of related genes as values. Now, please modify it \n",
    "# to a new dictionary with the same traits as keys, while the value is a dictionary with key 'related_genes' storing the orignial\n",
    "# list of genes, and key 'conditions' storing the list of conditions corresponding to the trait. \n",
    "# DO NOT save to any file yet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new dictionary to store both genes and conditions for each trait\n",
    "new_task_info = {}\n",
    "\n",
    "# Iterate through each trait in t2g\n",
    "for trait in t2g:\n",
    "    # Get the conditions for this trait from the selected DataFrame\n",
    "    trait_conditions = selected.loc[selected['Trait'] == trait, 'Condition'].unique().tolist()\n",
    "    \n",
    "    # Create the new dictionary structure for this trait\n",
    "    new_task_info[trait] = {\n",
    "        'related_genes': t2g[trait],  # Original list of genes\n",
    "        'conditions': trait_conditions       # List of conditions\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'related_genes': ['TP53',\n",
       "  'EGFR',\n",
       "  'TNF',\n",
       "  'IL6',\n",
       "  'VEGFA',\n",
       "  'TGFB1',\n",
       "  'MTHFR',\n",
       "  'HIF1A',\n",
       "  'ERBB2',\n",
       "  'ESR1',\n",
       "  'IL10',\n",
       "  'APP',\n",
       "  'STAT3',\n",
       "  'BRCA1',\n",
       "  'ACE',\n",
       "  'KRAS',\n",
       "  'VDR',\n",
       "  'MMP9',\n",
       "  'CD274',\n",
       "  'CRP',\n",
       "  'ADIPOQ',\n",
       "  'AKT1',\n",
       "  'ABCB1',\n",
       "  'NFKB1',\n",
       "  'IL1B',\n",
       "  'CTNNB1',\n",
       "  'PTEN',\n",
       "  'CDKN2A',\n",
       "  'TLR4',\n",
       "  'PTGS2',\n",
       "  'TERT',\n",
       "  'MYC',\n",
       "  'CXCL8',\n",
       "  'MTOR',\n",
       "  'PPARG',\n",
       "  'CDH1',\n",
       "  'IGF1',\n",
       "  'HLA-B',\n",
       "  'LEP',\n",
       "  'BCL2',\n",
       "  'BRCA2',\n",
       "  'CXCR4',\n",
       "  'NFE2L2',\n",
       "  'JAK2',\n",
       "  'MDM2',\n",
       "  'GSTM1',\n",
       "  'IL17A',\n",
       "  'MMP2',\n",
       "  'SIRT1',\n",
       "  'CCND1',\n",
       "  'MIR21',\n",
       "  'CCL2',\n",
       "  'PIK3CA',\n",
       "  'HMGB1',\n",
       "  'CDKN1A',\n",
       "  'NPPB',\n",
       "  'CTLA4',\n",
       "  'CD44',\n",
       "  'TLR2',\n",
       "  'BIRC5',\n",
       "  'NOTCH1',\n",
       "  'EZH2',\n",
       "  'ATM',\n",
       "  'GSTT1',\n",
       "  'MET',\n",
       "  'GSTP1',\n",
       "  'CXCL12',\n",
       "  'NR3C1',\n",
       "  'PDCD1',\n",
       "  'YAP1',\n",
       "  'SPP1',\n",
       "  'NLRP3',\n",
       "  'HMOX1',\n",
       "  'KIT',\n",
       "  'RELA',\n",
       "  'PARP1',\n",
       "  'XRCC1',\n",
       "  'GSK3B',\n",
       "  'HLA-G',\n",
       "  'FTO',\n",
       "  'FAS',\n",
       "  'IGF1R',\n",
       "  'MAPK14',\n",
       "  'ITGB1',\n",
       "  'MUC1',\n",
       "  'LCN2',\n",
       "  'MLH1',\n",
       "  'NOD2',\n",
       "  'ITGB3',\n",
       "  'ABCG2',\n",
       "  'IDH1',\n",
       "  'MIR146A',\n",
       "  'CYP1A1',\n",
       "  'HBB',\n",
       "  'KDR',\n",
       "  'MIF',\n",
       "  'CDKN1B',\n",
       "  'JUN',\n",
       "  'MIR155',\n",
       "  'TNFSF10',\n",
       "  'ALK',\n",
       "  'MGMT',\n",
       "  'FLT3',\n",
       "  'PTK2',\n",
       "  'STAT1',\n",
       "  'BAX',\n",
       "  'NPM1',\n",
       "  'EP300',\n",
       "  'SCN5A',\n",
       "  'RUNX1',\n",
       "  'WT1',\n",
       "  'UGT1A1',\n",
       "  'HSP90AA1',\n",
       "  'HGF',\n",
       "  'ALDH2',\n",
       "  'KCNH2',\n",
       "  'FGF2',\n",
       "  'FN1',\n",
       "  'TLR9',\n",
       "  'AHR',\n",
       "  'PTPN11',\n",
       "  'ALB',\n",
       "  'HDAC1',\n",
       "  'CASP8',\n",
       "  'HBA1',\n",
       "  'E2F1',\n",
       "  'AURKA',\n",
       "  'ERCC2',\n",
       "  'IL2',\n",
       "  'CDK1',\n",
       "  'NAT2',\n",
       "  'MIR34A',\n",
       "  'ABL1',\n",
       "  'MSH2',\n",
       "  'PLK1',\n",
       "  'HRAS',\n",
       "  'MAPK8',\n",
       "  'FGFR2',\n",
       "  'PRKCA',\n",
       "  'RAD51',\n",
       "  'FOXO3',\n",
       "  'MPO',\n",
       "  'MCL1',\n",
       "  'MALAT1',\n",
       "  'SQSTM1',\n",
       "  'EGF',\n",
       "  'FOXO1',\n",
       "  'ERCC1',\n",
       "  'BSG',\n",
       "  'TWIST1',\n",
       "  'FGFR1',\n",
       "  'IGFBP3',\n",
       "  'APEX1',\n",
       "  'DNMT1',\n",
       "  'TNFSF11',\n",
       "  'DPP4',\n",
       "  'NF1',\n",
       "  'POU5F1',\n",
       "  'OGG1',\n",
       "  'FOXM1',\n",
       "  'PLAUR',\n",
       "  'GNAS',\n",
       "  'CREBBP',\n",
       "  'ADAMTS13',\n",
       "  'TP73',\n",
       "  'MIR145',\n",
       "  'DICER1',\n",
       "  'VIM',\n",
       "  'FASLG',\n",
       "  'BCL2L1',\n",
       "  'PCNA',\n",
       "  'KMT2A',\n",
       "  'GRN',\n",
       "  'MEFV',\n",
       "  'PKM',\n",
       "  'CYP2E1',\n",
       "  'RUNX2',\n",
       "  'CDK2',\n",
       "  'CREB1',\n",
       "  'G6PD',\n",
       "  'IDO1',\n",
       "  'NQO1',\n",
       "  'HOTAIR',\n",
       "  'CALR',\n",
       "  'MICA',\n",
       "  'H19',\n",
       "  'STK11',\n",
       "  'PPARA',\n",
       "  'PIK3R1',\n",
       "  'SNAI1',\n",
       "  'BMI1',\n",
       "  'CDC42',\n",
       "  'DNMT3A',\n",
       "  'ELAVL1',\n",
       "  'TGM2',\n",
       "  'TNFSF13B',\n",
       "  'IL2RA',\n",
       "  'NRAS',\n",
       "  'EPAS1',\n",
       "  'APOBEC3G'],\n",
       " 'conditions': ['Age',\n",
       "  'Gender',\n",
       "  'Cervical_Cancer',\n",
       "  'Pancreatic_Cancer',\n",
       "  'Bladder_Cancer',\n",
       "  'Glioblastoma',\n",
       "  'Type_2_Diabetes',\n",
       "  'Hypertension',\n",
       "  'Obesity']}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_task_info['Acute_Myeloid_Leukemia']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"../metadata/task_info.json\", \"w\") as f:\n",
    "    json.dump(new_task_info, f, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"../metadata/task_info.json\", \"r\") as f:\n",
    "    new_task_info2 = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_task_info == new_task_info2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(new_task_info['Acute_Myeloid_Leukemia'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:37:52.342393202Z",
     "start_time": "2024-06-04T19:37:52.313074846Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "thres = 795  # tried out\n",
    "selected = rel[(rel.index < thres) | rel['Condition'].isin(condition_all)]\n",
    "selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:41:52.444797268Z",
     "start_time": "2024-06-04T19:41:52.441682039Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "selected[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:38:02.323492700Z",
     "start_time": "2024-06-04T19:38:02.313404557Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "selected[(selected['Trait'].isin(cancer_traits)) & (selected['Condition'].isin(cancer_traits))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:38:09.740599039Z",
     "start_time": "2024-06-04T19:38:09.728529082Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "selected[selected['Condition'] == 'Age']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-04T19:38:10.694802090Z",
     "start_time": "2024-06-04T19:38:10.680980752Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "selected[selected['Condition'] == 'Gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top1k = rel.iloc[:1000]\n",
    "top1k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#rel[rel['Trait'] == 'Longevity and Aging' & rel['Condition'] == 'Age']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#rel[(rel['Trait'] == 'Longevity and Aging') & (rel['Condition'] == 'Age')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top1k[top1k['Condition'] == 'Obesity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top1k[top1k['Condition'] == 'Hypertension']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top1k[top1k['Condition'] == 'Bipolar disorder']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top1k[top1k['Condition'] == 'Anxiety disorder']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top1k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all = pd.read_csv('new_task.csv')\n",
    "all_cancers = all.loc[all[\"Type\"] == \"1. Cancer and Oncology-Related Disorders\", 'Trait'].tolist()\n",
    "all_cancers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cancer_traits = [c for c in traits if c in all_cancers]\n",
    "len(cancer_traits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(cancer_traits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top1k[top1k['Trait'].isin(cancer_traits) | top1k['Condition'].isin(cancer_traits)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top1k[top1k['Trait'].isin(cancer_traits) & top1k['Condition'].isin(cancer_traits)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "c\"\"\"for file in sorted_file_names:\n",
    "    file_path = os.path.join('downloaded_ncbi_genes', file)\n",
    "    with open(file_path, 'r') as f:\n",
    "        if \"<!-- download operation failed -->\" in f.read():\n",
    "            print(file)\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_excel(\"Trait_Condition_Pairings_v2.xlsx\", sheet_name=\"trait and condition\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "traits = pd.read_csv(\"latest_task.csv\")['Trait'].tolist()\n",
    "len(traits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "traits.remove('Vitamin D Levels')\n",
    "len(traits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "values = traits + ['Age', 'Gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rel = df.loc[df[843] > 2, ['Trait', 'Condition']]\n",
    "rel = rel.drop_duplicates(keep='last')\n",
    "rel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rel[~rel['Trait'].isin(traits) & ~rel['Condition'].isin(values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_excel(\"Trait_Condition_Pairings_v2.xlsx\", sheet_name=\"clusters\")\n",
    "df.isin(traits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agent",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
